package com.chance.cc.crawler.development.bootstrap.jingdong;

import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.downloader.proxy.Proxy;
import com.chance.cc.crawler.core.filter.FilterUtils;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.development.controller.DevCrawlerController;
import org.apache.commons.lang3.StringUtils;

import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerDataType.comment;
import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerRequestType.*;

/**
 * @author bx
 * @date 2020/11/11 0011 10:44
 */
public class JDCommentStart {

    public static final String domainId = "jingdong";

    private static Proxy proxy = new Proxy();
    static {
        //代理配置
        proxy.setHost("http-dyn.abuyun.com");
        proxy.setPort(9020);
        proxy.setUsername("HEW657EL99F83S9D");
        proxy.setPassword("8916B1F3F10B1979");
    }

    public static void main(String[] args) {

//        CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
//                .startPageRequest(domainId, CrawlerEnum.CrawlerRequestType.turnPage)
//                .httpUrl("https://club.jd.com/comment/productPageComments.action")
//                .releaseTime(System.currentTimeMillis())
//                .needWashed(false)
//                .needParsed(false)
//                .resultLabelTag(comment)
//                .filter(CrawlerEnum.CrawlerRecordFilter.keyOrDateRange)
//                .addFilterInfo(FilterUtils.memoryFilterKeyInfo(domainId))
//                .addFilterInfo(FilterUtils.dateRangeFilterInfo(24*1,null))
//                .proxy(proxy)
//                .build();
//        requestRecord.setDownload(false);
//        requestRecord.setSkipPipeline(true);

        CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domainId, CrawlerEnum.CrawlerRequestType.turnPage)
                .httpUrl("https://club.jd.com/comment/productPageComments.action?callback=fetchJSON_comment98" +
                        "&productId=11869314310&score=0&sortType=6&page=0&pageSize=10&isShadowSku=0&fold=1")
                .releaseTime(System.currentTimeMillis())
                .needWashed(true)
                .resultLabelTag(comment)
                .filter(CrawlerEnum.CrawlerRecordFilter.keyOrDateRange)
                .addFilterInfo(FilterUtils.memoryFilterKeyInfo(domainId))
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(24*1,null))
                .proxy(proxy)
                .build();

        //初始 comment crawler request record
        String requestQueueName = StringUtils.joinWith("-","crawler",domainId,"comment","queue");

        //user agents source record
        CrawlerRequestRecord userAgentsCrawlerRecord = CrawlerRequestRecord.builder()
                .startPageRequest("taobao_comment_user_agents", turnPageItem)
                .httpUrl("https://fake-useragent.herokuapp.com/browsers/0.1.11")
                .requestLabelTag(supportSource)
                .requestLabelTag(internalDownload)
                .build();

        DevCrawlerController.builder()
                .crawlerThreadNum(1)
                .triggerInfo(domainId,domainId,System.currentTimeMillis(),domainId)
                .crawlerRequestQueue(DevCrawlerController.devRequestQueue(requestQueueName))
                .consoleResultPipeline()
                .fileResultPipeline("D:\\chance\\log\\jingdong\\jingdong_comment.log",false)
                .requestRecord(requestRecord)
                .supportRecord(userAgentsCrawlerRecord)
                .build().start();
    }
}
